library(tm)
library(SnowballC)


dtm.text<-paste(df_sample$Title, df_sample$Body, sep=" ")

dtm.text<-tolower(dtm.text)
dtm.text<- gsub("<code>.*</code>"," ",dtm.text, perl=T)
dtm.text<- gsub("<p>"," ",dtm.text, fixed=T)
dtm.text<- gsub("</p>"," ",dtm.text, fixed=T)
dtm.text<- gsub("\n"," ",dtm.text, fixed=T)
dtm.text<- gsub("<strong>"," ",dtm.text,fixed=T)
dtm.text<- gsub("<strong>"," ",dtm.text)
dtm.text<- gsub("</strong>"," ",dtm.text)
dtm.text<- gsub("<p>"," ",dtm.text,fixed=T)
dtm.text<- gsub("<pre>"," ",dtm.text,fixed=T)
dtm.text<- gsub("</pre>"," ",dtm.text,fixed=T)
dtm.text<- gsub("\\(.*\\)","",dtm.text,perl=T)
dtm.text<- gsub("c#","csharp",dtm.text, fixed=T)               ##So c# and c++ will be recognized as term
dtm.text<- gsub("f#","fsharp",dtm.text, fixed=T)
dtm.text<- gsub("c++","cplusplus",dtm.text, fixed=T)
dtm.text<-gsub("[[:punct:]]", " ", dtm.text, perl=T)


text.corpus<-tm_map(Corpus(VectorSource(dtm.text)),tolower)

mystops<-c(
 "a","about","above","after","again","against","all","am",
"an","and","any","appreciate","aren't","are","as","at",
"be","because","been","before","being","below","between",
"both","but","by","can't","cannot","couldn't","could",
"didn't","did","do","doesn't","does","doing","don't",
"down","during","each","few","for","from","further",
"hadn't","had","hasn't","has","haven't","have","having",
"he'd","he'll","he's","he","help","her","here's","here",
"hers","herself","him","himself","his","how's","how",
"i'd","i'll","i'm","i've","i","if","in","into","isn't",
"is","it's","it","its","itself","let's","me","more",
"most","mustn't","my","myself","no","nor","not","of",
"off","on","once","only","or","other","ought","our",
"ours ","ourselves","out","over","own","same","shan't",
"she'd","she'll","she's","shouldn't","should","she","so",
"some","such","than","thank","thanks","that's","that","the",
"theirs","their","them","themselves","then","there's","there",
"these","they'd","they'll","they're","they've","they",
"this","those","through","to","too","under","until","up",
"very","wasn't","was","we'd","we'll","we're","we've",
"we","were","weren't","what's","what","when's","when",
"where's","where","which","while","who's","who","whom",
"why's","why","with","won't","wouldn't","would","you'd",
"you'll","you're","you've","you","your","yours","yourself",
"your","application", "can", "code", "create", "data", "error",
"find", "following", "get", "just", "know","like", "need", "pre",
"use", "using", "want", "way", "will","now", "one", "problem",
 "something", "sure", "trying", "work", "right", "run", "running",
 "see", "seems", "set", "show", "similar", "simple", "since",
 "possible",  "value","method","also","app","time","works" , "please"
)

text.corpus<-tm_map(text.corpus, removeWords, mystops )
text.corpus<-tm_map(text.corpus, removeWords, stopwords("english"))
text.corpus<-tm_map(text.corpus, stripWhitespace)

##USE SEPARATE VARIABLES FOR TERM FREQUENCY IN TITLE VERSUS BODY? CODE TOO?
##DOES > 25 THRESHOLD HELP OR NOT?

dtm <- removeSparseTerms(DocumentTermMatrix(text.corpus), 0.999)
